Loading required package: Rcpp
Loading 'brms' package (version 2.18.0). Useful instructions
can be found by typing help('brms'). A more detailed introduction
to the package is available through vignette('brms_overview').
Attaching package: 'brms'
The following object is masked from 'package:lme4':
ngrps
The following object is masked from 'package:stats':
ar
library(viridis)
Loading required package: viridisLite
library(ggdist)
Attaching package: 'ggdist'
The following objects are masked from 'package:brms':
dstudent_t, pstudent_t, qstudent_t, rstudent_t
library(tidybayes)
Attaching package: 'tidybayes'
The following objects are masked from 'package:brms':
dstudent_t, pstudent_t, qstudent_t, rstudent_t
# wrangle information on the plot type, ES, ...plot_info <- study1 %>%pivot_longer(2:195, names_to ="variables", values_to ="values", values_transform = as.character) %>% dplyr::filter(str_detect(variables, "plot")) %>%# we only need the rows with info on plots tidyr::separate(col = values, into =c("type", "axis", "effsize"), # separate the info into three columnssep ="_", remove = F) %>% dplyr::mutate(plot = variables, # rename variables for later jointype =paste(type, axis, sep ="_")) %>% dplyr::select(-variables, -axis)# wrangle answers to items on each pageitem_values <- study1 %>% dplyr::select(-c(topic:itemo)) %>%pivot_longer(2:169, names_to ="variables", values_to ="values", values_transform = as.character) %>% dplyr::mutate(variables =case_when( # recode variable names that have variables =="sensi_6"~"sensi_06", # accidentally been labeled variables =="acccl_6"~"acccl_06", # without zero variables =="accu3_6"~"accu3_06", variables =="accov_6"~"accov_06", variables =="diffi_6"~"diffi_06", variables =="infor_6"~"infor_06", variables =="value_6"~"value_06",TRUE~ variables )) %>% dplyr::mutate(plot =paste0("plotx_", str_sub(variables, -2, -1)), # create variable for later joinvariables =str_sub(variables, 1, -4)) %>%# rename variable names to get a data set # with one line per participant per pagepivot_wider(id_cols =c(session, plot), names_from ="variables", values_from ="values")# join the two data setsstudy1_w <-full_join(plot_info, item_values, by =c("session", "plot")) %>%# by participant and page (plot) dplyr::select(-values) %>% dplyr::mutate(rating_cl =as.numeric(acccl), # some var need to be defined asrating_u3 =as.numeric(accu3), # numeric againrating_ov =as.numeric(accov),diffi =as.numeric(diffi),infor =as.numeric(infor),value =as.numeric(value),effsize =as.numeric(effsize),effsize_cl =case_when( # there is no negative Cliff's Delta, so we have to compute # two transformations effsize >0~ (((2*pnorm(effsize/2))-1)/pnorm(effsize/2)),# transform the actual effect size Cohen's d to Cliff's Delta effsize <0~ (- (((2*pnorm(abs(effsize)/2))-1)/pnorm(abs(effsize)/2))) # transform the actual effect size Cohen's d to Cliff's Delta # and make it negative as in the item ),effsize_u3 =1-pnorm(effsize), # reverse so that it fits the direction of the U3 item# transform the actual effect size Cohen's d to Cohen's U3effsize_ov =2*pnorm(-abs(effsize) /2), # transform the actual effect size Cohen's d to overlap# actual difference of rating relative to depicted effectsize diff_cl = (rating_cl - effsize_cl)/2,# actual difference of rating relative to depicted effectsizediff_u3 = (rating_u3/100) - effsize_u3,# actual difference of rating relative to depicted effectsize diff_ov = (rating_ov/100) - effsize_ov,diffi_normed = ((diffi -1) /3) -1, # transform item to -1 to 1infor_normed = ((infor -1) /3) -1, # transform item to -1 to 1value_normed = ((value -1) /3) -1) %>%# transform item to -1 to 1group_by(session) %>%mutate(rating_ov_missconcept =median(rating_ov, na.rm = T) <68.9,rating_u3_missconcept =median(rating_u3, na.rm = T) <21.2) %>%ungroup() %>%mutate(rating_u3_filtered =ifelse(rating_u3_missconcept == T, NA, rating_u3),rating_ov_filtered =ifelse(rating_ov_missconcept == T, NA, rating_ov),diff_u3_filtered = (rating_u3_filtered/100) - effsize_u3,diff_ov_filtered = (rating_ov_filtered/100) - effsize_ov,sensi_binary =ifelse(is.na(sensi), # 1 if NOT "equal"NA,as.numeric(!grepl("equal", sensi))),sensi_ordinal =ordered(factor(substr(sensi, 55, 100)),levels =c("inferior","equal","superior")),sensi_binary_filtered =case_when(sensi_ordinal =="equal"~0, (sensi_ordinal =="inferior"& effsize <0) | (sensi_ordinal =="superior"& effsize >0) ~as.numeric(NA),TRUE~1),effsize_abs =abs(effsize))# create a list of u3_misconceptualizersu3_misconceptualizers <- study1_w %>%filter(rating_u3_missconcept == T) %>%pull(session) %>%unique()# create a list of ov_misconceptualizersov_misconceptualizers <- study1_w %>%filter(rating_ov_missconcept == T) %>%pull(session) %>%unique() ### wrangle time stamp data ####################################################study1_w_timestamp <-read_csv("../data/teachers_study1_N40_detailed.csv") %>%# filter participants from study1_w onlyfilter(session %in% study1_w$session) %>%# we only need vars sensitivity or accuracy dplyr::filter(str_detect(item_name, "sensi|acccl|accu3|accov")) %>%# create var with plot numbermutate(plot =paste0("plotx_", str_sub(item_name, -2, -1)),# recode wrong item labellingplot =ifelse(plot =="plotx__6", "plotx_06", plot)) %>%relocate(session, plot) %>%# delete the page number in item namemutate(item_name =str_sub(item_name, 1, 5)) %>%pivot_wider(id_cols =c(session, plot), names_from = item_name, values_from = answered_relative) %>%rowwise() %>%# what was the time of the first item to be clicked?mutate(effic =min(sensi, acccl, accu3, accov, na.rm=T)) %>%ungroup() %>% dplyr::select(session, plot, effic, sensi, acccl, accu3, accov) %>%left_join(., study1_w %>%select(session, plot, type), by=c("session", "plot")) %>%# generate data set so that the six plots from the same type are ordered# one after the other (and not 1-24)group_by(session, type) %>%arrange(plot) %>%mutate(plotNrWithin =1:n()) %>%ungroup() %>%group_by(plotNrWithin, type) %>%mutate(effic_10righttrunc =ifelse(effic >quantile(effic, .9), NA, effic),effic_05righttrunc =ifelse(effic >quantile(effic, .95), NA, effic),log_effic_05righttrunc =log(effic_05righttrunc),log_effic_10righttrunc =log(effic_10righttrunc),plotNrWithin0 = plotNrWithin -1,plotNrWithin_factor =as.factor(plotNrWithin)) %>%ungroup()
Rows: 27336 Columns: 14
── Column specification ────────────────────────────────────────────────────────
Delimiter: ","
chr (3): session, item_name, answer
dbl (7): unit_session_id, item_id, shown_relative, answered_relative, displ...
dttm (4): created, saved, shown, answered
ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
Somewhat disturbing is the first mode in rating_ov. Maybe some users confused overlap and non-overlap? Another artifact seems to be the first mode in rating_u3.
pp_check(logreg_mod2) +theme_modern_rc() +scale_color_manual(values=viridis(2, begin = .3))
Using 10 posterior draws for ppc type 'dens_overlay' by default.
Scale for colour is already present.
Adding another scale for colour, which will replace the existing scale.
Efficiency
Visualisation
Raw data
ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic)) +geom_boxplot(alpha = .2, color ="lightgrey") +geom_sina(alpha = .5) +coord_cartesian(ylim =c(0,100000)) +facet_wrap(~type) +theme_modern_rc() +labs(title ="Dwell Times Until First Decision",subtitle ="Per Plot Type and Plot Repetition") +theme(strip.text =element_text(color ="white"))
5% Percent Truncated
ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic_05righttrunc)) +geom_boxplot(alpha = .2, color ="lightgrey") +geom_sina(alpha = .5) +coord_cartesian(ylim =c(0,85000)) +facet_wrap(~type) +theme_modern_rc() +labs(title ="5% Truncated Dwell Times Until First Decision",subtitle ="Per Plot Type and Plot Repetition") +theme(strip.text =element_text(color ="white"))
ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), log(effic_05righttrunc))) +geom_boxplot(alpha = .2, color ="lightgrey") +geom_sina(alpha = .5) +facet_wrap(~type) +theme_modern_rc() +labs(title ="log Transformed Dwell Times Until First Decision",subtitle ="Per Plot Type and Plot Repetition") +theme(strip.text =element_text(color ="white"))
Estimated Bayes factor in favor of plot0306_mod2 over plot0306_mod1: 18618875449292113148499011020035359633188591249536319488.00000
Research Question 2
Contrast Effects
Graphical Overview
# preparing multilevel model with brms for graphical overview# rating u3mod_plot_ratingu3 <-brm(scale(rating_u3) ~ (1| type),data = study1_w,control =list(adapt_delta = .999),cores =4,iter =20000)
Warning: Rows containing NAs were excluded from the model.
Compiling Stan program...
Trying to compile a simple C file
Start sampling
Warning: There were 4 divergent transitions after warmup. See
https://mc-stan.org/misc/warnings.html#divergent-transitions-after-warmup
to find out why this is a problem and how to eliminate them.
Warning: Examine the pairs() plot to diagnose sampling problems
Warning: `gather_()` was deprecated in tidyr 1.2.0.
ℹ Please use `gather()` instead.
ℹ The deprecated feature was likely used in the tidybayes package.
Please report the issue at <]8;;https://github.com/mjskay/tidybayes/issues/newhttps://github.com/mjskay/tidybayes/issues/new]8;;>.
plot_ratingu3
Warning: Using the `size` aesthietic with geom_segment was deprecated in ggplot2 3.4.0.
ℹ Please use the `linewidth` aesthetic instead.
# efficiency for the first plotmod_plot_efficiency1 <-brm(scale(effic) ~ (1| type),data = study1_w_timestamp_effsize %>%filter(plotNrWithin ==1),control =list(adapt_delta = .999),cores =4 )
Compiling Stan program...
Trying to compile a simple C file
Start sampling
Warning: There were 3 transitions after warmup that exceeded the maximum treedepth. Increase max_treedepth above 10. See
https://mc-stan.org/misc/warnings.html#maximum-treedepth-exceeded
Warning: Examine the pairs() plot to diagnose sampling problems
Warning: Bulk Effective Samples Size (ESS) is too low, indicating posterior means and medians may be unreliable.
Running the chains for more iterations may help. See
https://mc-stan.org/misc/warnings.html#bulk-ess
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
Warning: Bulk Effective Samples Size (ESS) is too low, indicating posterior means and medians may be unreliable.
Running the chains for more iterations may help. See
https://mc-stan.org/misc/warnings.html#bulk-ess
Warning: Tail Effective Samples Size (ESS) is too low, indicating posterior variances and tail quantiles may be unreliable.
Running the chains for more iterations may help. See
https://mc-stan.org/misc/warnings.html#tail-ess
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
Warning: There were 1 divergent transitions after warmup. See
https://mc-stan.org/misc/warnings.html#divergent-transitions-after-warmup
to find out why this is a problem and how to eliminate them.
Warning: Examine the pairs() plot to diagnose sampling problems
Arslan, Ruben C., Matthias P. Walther, and Cyril S. Tata. 2020. “Formr: A Study Framework Allowing for Automated Feedback Generation and Complex Longitudinal Experience-Sampling Studies Using R.”Behavior Research Methods 52 (1): 376–87. https://doi.org/10.3758/s13428-019-01236-y.